# -*-encoding:utf-8-*-
import requests
import building
from bs4 import BeautifulSoup
import csv
import threading
from time import gmtime, strftime

time_begin=gmtime()

def log(data):
	with open('51nb.txt', 'a') as f:
		f.write(data)

with open('51nb.csv', 'wb') as f:
	f.write(u'\ufeff'.encode('utf8'))

titles=[u'板块',u'交易主题',u'商品价格',u'商家/会员',u'卖家资料',u'页面链接',u'交易地点',u'发布时间',u'回复帖子',u'最后回复时间',u'最后回复ID',u'备注']
with open('51nb.csv', 'ab+') as f:
	writer = csv.writer(f)
	writer.writerow([title.encode('utf8') for title in titles])

host='https://forum.51nb.com/'
lock = threading.Lock()
rows=[]
threads=[]

def get_row(trs,title):	
	rs=[]
	for tr in trs:
		row=[title]
		tds=tr.find_all('td')
		row.append(tds[0].find('a').string)
		row.append(tds[1].string)
		row.append(tds[2].find('a').string)
		row.append(host+tds[2].find('a')['href'])
		row.append(host+tds[0].find('a')['href'])
		
		remark=tds[0]['title'].split('\n')
		row.append(remark[1].split(u'：')[1])
		row.append(remark[3].split(u'：')[1])
		row.append(remark[4].split(u'：')[1])
		row.append(remark[5].split(u'：')[1].split(u' by ')[0])
		row.append(remark[5].split(u' by ')[1])
		row.append(tds[0]['title'])
		rs.append(row)
		#break
	return rs

def log_forum(page):
	url=host+'forum.php?mod=forumdisplay&fid=41&page='+str(page)+'&orderby=dateline'
	r=requests.get(url, stream=True)
	html=r.content#.decode('gbk').encode('utf-8')

	soup =building.BeautifulSoup(html, 'html.parser')
	#print soup.prettify()
	#log(soup.prettify('utf-8'))
	moderate=soup.find_all(id='moderate')[0]#帖子板块
	tables=moderate.find_all('table')
	business=tables[1]#商家商品
	member=tables[2]#会员商品
	b_trs=business.find_all('tr',bgcolor=True)
 	rs=get_row(b_trs,u'商家商品')
	m_trs=member.find_all('tr',bgcolor=True)
	rs2=get_row(m_trs,u'会员商品')
	rs+=rs2
	return rs
			 

def run_thread(page):
	print 'thread %s >> begin'%(threading.current_thread().name)
	r=log_forum(page)
	print 'thread %s >> lock with %s rows'%(threading.current_thread().name,len(r))
	global rows
	print 'thread %s >> locked '%(threading.current_thread().name)
    # 先要获取锁:
	lock.acquire()
	try:
		rows+=r
	finally:
		lock.release()
	print 'thread %s >> lock release'%(threading.current_thread().name)

for num in range(3):
	t=threading.Thread(target=run_thread,name='t'+str(num),args=(num,))	
	threads.append(t)
	t.start()
	#t.join()
for t in threads:
	t.join()

for row in rows:
	for r in row:
		pass#print r

with open('51nb.csv', 'ab+') as f:
	writer = csv.writer(f)
	for row in rows:
		writer.writerow([r.encode('utf8') for r in row])

time_end=gmtime()
print 'time_begin:'+strftime("%Y %m %d %H:%M:%S", time_begin)
print 'time_end:'+strftime("%Y %m %d %H:%M:%S", time_end) 
# 'zone='+str